import time
from scrapy import cmdline
from multiprocessing import Pool

if __name__ == '__main__':
    spiderList = [
        "airspacemag",
        "breakingdefense_air",
        "breakingdefense_allies",
        "breakingdefense_congress",
        "breakingdefense_land",
        "breakingdefense_networks_and_cyber",
        "breakingdefense_pentagon",
        "breakingdefense_space",
        "breakingdefense_threats",
        "c4isr",
        "caltech_news",
        "cmu",
        "esa_Science_Exploration",
        "ethz",
        "gatech",
        "harvard",
        "isro",
        "jaxa_press",
        "jpl_nasa_news",
        "ll_news",
        "nasa_centers_glenn",
        "nasa_goddard",
        "nasa_news",
        "nationaldefensemagazine",
        "oxac",
        "princeton",
        "roscosmos_news",
        "space",
        "spacenews_civil",
        "spacenews_commercial",
        "spacenews_launch",
        "spacenews_military",
        "spacenews_news",
        "spacenews_opinion",
        "spacenews_policy_politics",
        "spacenews_webinars",
        "spacenewsfeed",
        "spacewar",
        "stanford",
        "ucla"
    ]
    p = Pool(5)
    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))
    for spider in spiderList:
        args = f'scrapy crawl {spider} -s LOG_FILE=log\\{spider}.log'
        print("运行爬虫:" + spider)
        p.apply_async(cmdline.execute, args=(args.split(),))
    p.close()
    # print("爬虫结束:")
    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))
    p.join()
